Fully Convolutional Network¶
In [1]:
# Imports
import os
from keras.layers import *
from keras.models import Model, load_model
from keras.regularizers import l2
from keras.callbacks import ModelCheckpoint
from keras.optimizers import RMSprop, Adam, SGD
import matplotlib.pyplot as plt
import scipy.misc as m
FCN model¶
In [2]:
def FCN32(d=0.2):
inp = Input(shape=(None,None,3))
# Block 1
c11 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1', kernel_regularizer=l2(5e-4))(inp)
c12 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2', kernel_regularizer=l2(5e-4))(c11)
p1 = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(c12)
# Block 2
c21 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1', kernel_regularizer=l2(5e-4))(p1)
c22 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2', kernel_regularizer=l2(5e-4))(c21)
p2 = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(c22)
# # # Block 3
# c31 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1', kernel_regularizer=l2(5e-4))(p2)
# c32 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2', kernel_regularizer=l2(5e-4))(c31)
# c33 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3', kernel_regularizer=l2(5e-4))(c32)
# p3 = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(c33)
# # Block 4
# c41 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1', kernel_regularizer=l2(5e-4))(p3)
# c42 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2', kernel_regularizer=l2(5e-4))(c41)
# c43 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3', kernel_regularizer=l2(5e-4))(c42)
# p4 = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(c43)
# # Block 5
# c51 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1', kernel_regularizer=l2(5e-4))(p4)
# c52 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2', kernel_regularizer=l2(5e-4))(c51)
# c53 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3', kernel_regularizer=l2(5e-4))(c52)
# p5 = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(c53)
# Convolutional layers transfered from fully-connected layers
# fc1 = Conv2D(1024, (7, 7), activation='relu', padding='same', name='fc1')(p5)
# fc1 = Dropout(d)(fc1)
# fc2 = Conv2D(1024, (1, 1), activation='relu', padding='same', name='fc2')(fc1)
# fc2 = Dropout(d)(fc2)
#classifying layer
# out = Conv2D(2, (1, 1), kernel_initializer='he_normal', activation='linear', padding='valid', strides=(1, 1))(p2)
# output = Conv2DTranspose(2, (32,32), activation='softmax', strides=(32, 32))(out)
output = UpSampling2D(size=(4,4))(p2)
output = Conv2D(2,(1,1), activation='softmax')(output)
model = Model(inputs=inp, outputs=output)
return model
In [3]:
fcn32s = FCN32()
fcn32s.summary()
DRIVE¶
Read training and validation sets¶
In [4]:
dir_drive = "/home/augustocms/datasets/drive/training/"
dir_img = dir_drive + "images/"
dir_seg = dir_drive + "1st_manual/"
dir_test = "/home/augustocms/datasets/drive/test/"
dir_test_img = dir_test + "images/"
dir_test_seg = dir_test + "1st_manual/"
with open(dir_drive + 'train.txt') as f:
x = f.read().splitlines()
imgs = np.array(x)
with open(dir_drive + 'val.txt') as f:
x = f.read().splitlines()
val_imgs = np.array(x)
with open(dir_test + 'test.txt') as f:
x = f.read().splitlines()
test_imgs = np.array(x)
print(imgs[0])
Load data¶
In [5]:
def extract_patches(img, size):
H, W, *c = img.shape
h = H//size[0]
w = W//size[1]
patches = []
for i in range(h):
for j in range(w):
patches.append(img[i*size[0]:(i+1)*size[0], j*size[1]:(j+1)*size[1]])
return np.array(patches)
def to_categorical(patches):
n, H, W = patches.shape
segmentation = np.zeros((n,H,W,2))
for i in range(n):
segmentation[i,:,:,0] = (patches[i,:,:] == 0).astype(int)
segmentation[i,:,:,1] = (patches[i,:,:] > 0).astype(int)
return segmentation
In [6]:
images = []
gt = []
for line in imgs:
names = line.split(' ')
images.append(m.imread(dir_img + names[0]))
gt.append(m.imread(dir_seg + names[1]))
images = np.array(images)
In [7]:
patch_size = 32
X_train = []
y_train = []
for i, img in enumerate(images):
seg_img = gt[i]
X_train.append(extract_patches(img, (patch_size, patch_size)))
y_patches = to_categorical(extract_patches(seg_img, (patch_size,patch_size)))
y_train.append(y_patches)
fig = plt.figure(figsize=(20,40))
ax = fig.add_subplot(1,2,1)
ax.imshow(img)
ax.set_title("original image")
ax = fig.add_subplot(1,2,2)
ax.imshow(seg_img, cmap='gray')
ax.set_title('ground truth')
plt.show()
In [8]:
X_train = np.array(X_train)
X_train = np.reshape(X_train, (-1, patch_size,patch_size,3))
y_train = np.array(y_train)
y_train = np.reshape(y_train, (-1, patch_size, patch_size,2))
print(X_train.shape, y_train.shape)
In [9]:
X_val = []
y_val = []
for line in val_imgs:
names = line.split(' ')
val_img = m.imread(dir_img + names[0])
val_seg = m.imread(dir_seg + names[1])
X_val.append(extract_patches(val_img, (patch_size, patch_size)))
y_patches = to_categorical(extract_patches(val_seg, (patch_size,patch_size)))
y_val.append(y_patches)
X_val, y_val = np.array(X_val), np.array(y_val)
X_val = np.reshape(X_val, (-1, patch_size, patch_size, 3))
y_val = np.reshape(y_val, (-1, patch_size, patch_size, 2))
print(X_val.shape, y_val.shape)
Checking if patch and ground truth matches:
In [10]:
n = 100
fig = plt.figure(figsize=(5,10))
ax = fig.add_subplot(1,2,1)
ax.imshow(X_train[n][:,:])
ax.set_title('img')
ax = fig.add_subplot(1,2,2)
ax.imshow(y_train[n][:,:,1], cmap='gray')
ax.set_title('gt')
plt.show()
Training¶
In [11]:
opt = Adam(lr=1e-4)
fcn32s.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
checkpoint = ModelCheckpoint('weights/fcn_drive.hdf5', monitor='val_loss', verbose=1, save_best_only=True, mode='min')
In [12]:
# fcn32s.fit(X_train, y_train,
# validation_data=(X_val, y_val),
# epochs=60, batch_size=8,
# verbose=2,
# callbacks=[checkpoint])
# fcn32s.save('models/fcn_drive.h5')
fcn32s = load_model('models/fcn_drive.h5')
Predictions on test set¶
In [13]:
X_test = []
y_test = []
for line in test_imgs:
names = line.split(' ')
test_img = m.imread(dir_test_img + names[0])
test_seg = m.imread(dir_test_seg + names[1])
X_test.append(test_img)
y_ing = to_categorical(np.array([test_seg]))
y_test.append(y_ing)
In [14]:
y_test = np.array(y_test)
y_test = np.squeeze(y_test)
X_test = np.array(X_test)
print(X_test.shape, y_test.shape)
In [15]:
y_pred = fcn32s.predict(X_test)
y_predi = np.argmax(y_pred, axis=3)
y_testi = np.argmax(y_test, axis=3)
print(y_testi.shape,y_predi.shape)
In [16]:
gtp = (y_testi == 1).astype(int)[:,:,:-1]
pp = (y_predi == 1).astype(int)
gtn = (y_testi == 0).astype(int)[:,:,:-1]
pn = (y_predi == 0).astype(int)
TP = (gtp*pp).sum()
TN = (gtn*pn).sum()
FP = (gtn*pp).sum()
FN = (gtp*pn).sum()
Precision = TP/(TP+FP)
Sensitivity = TP/(TP+FN)
FMeasure = 2 * (Precision*Sensitivity)/(Precision + Sensitivity)
print(Precision, Sensitivity, FMeasure)
In [17]:
shape = (584, 565)
n_classes= 1
for i in range(10):
img_is = X_test[i]
seg = y_predi[i]
segtest = y_testi[i]
fig = plt.figure(figsize=(20,40))
ax = fig.add_subplot(1,3,1)
ax.imshow(img_is, cmap='gray')
ax.set_title("original")
ax = fig.add_subplot(1,3,2)
ax.imshow(seg, cmap='gray')
ax.set_title("predicted class")
ax = fig.add_subplot(1,3,3)
ax.imshow(segtest, cmap='gray')
ax.set_title("true class")
plt.show()
m.imsave('images/drive/'+str(i)+'_orig.png', img_is)
m.imsave('images/drive/'+str(i)+'_pred_fcn.png', seg)
m.imsave('images/drive/'+str(i)+'_segm.png', segtest)
print(img_is.shape, seg.shape, segtest.shape)
DIBCO¶
Read training and validation sets¶
In [18]:
dir_dibco = "/home/augustocms/scratch/DIBCO/"
with open(dir_dibco + 'train.txt') as f:
x = f.read().splitlines()
imgs = np.array(x)
with open(dir_dibco + 'val.txt') as f:
x = f.read().splitlines()
val_imgs = np.array(x)
with open(dir_dibco + 'test.txt') as f:
x = f.read().splitlines()
test_imgs = np.array(x)
print(imgs[0])
Load data¶
In [19]:
images = []
gt = []
for line in imgs:
names = line.split()
images.append(m.imread(dir_dibco + names[0], mode='RGB'))
gt.append(m.imread(dir_dibco + names[1], mode='L'))
images = np.array(images)
gt = np.array(gt)
In [20]:
patch_size = 32
X_train = []
y_train = []
for i, img in enumerate(images):
seg_img = gt[i]
X_train.append(extract_patches(img, (patch_size, patch_size)))
y_patches = to_categorical(extract_patches(seg_img, (patch_size, patch_size)))
y_train.append(y_patches)
fig = plt.figure(figsize=(20,40))
ax = fig.add_subplot(1,2,1)
ax.imshow(img)
ax.set_title("original image")
ax = fig.add_subplot(1,2,2)
ax.imshow(seg_img, cmap='gray')
ax.set_title('ground truth')
plt.show()
In [21]:
X_train = np.array(X_train)
X_train = np.vstack(X_train)
X_train = np.reshape(X_train, (-1, patch_size,patch_size,3))
y_train = np.array(y_train)
y_train = np.vstack(y_train)
y_train = np.reshape(y_train, (-1, patch_size, patch_size,2))
print(X_train.shape, y_train.shape)
In [22]:
X_val = []
y_val = []
for line in val_imgs:
names = line.split()
val_img = m.imread(dir_dibco + names[0], mode='RGB')
val_seg = m.imread(dir_dibco + names[1], mode='L')
X_val.append(extract_patches(val_img, (patch_size, patch_size)))
y_patches = to_categorical(extract_patches(val_seg, (patch_size,patch_size)))
y_val.append(y_patches)
X_val, y_val = np.array(X_val), np.array(y_val)
X_val = np.vstack(X_val)
X_val = np.reshape(X_val, (-1, patch_size, patch_size, 3))
y_val = np.vstack(y_val)
y_val = np.reshape(y_val, (-1, patch_size, patch_size, 2))
print(X_val.shape, y_val.shape)
Checking if patch and ground truth matches:
In [23]:
n = 533
fig = plt.figure(figsize=(5,10))
ax = fig.add_subplot(1,2,1)
ax.imshow(X_train[n][:,:])
ax.set_title('img')
ax = fig.add_subplot(1,2,2)
ax.imshow(y_train[n][:,:,1], cmap='gray')
ax.set_title('gt')
plt.show()
Training¶
In [24]:
opt = Adam(lr=1e-4)
fcn32s.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
checkpoint = ModelCheckpoint('weights/fcn_dibco.hdf5', monitor='val_loss', verbose=1, save_best_only=True, mode='min')
In [25]:
# fcn32s.fit(X_train, y_train,
# validation_data=(X_val, y_val),
# epochs=60, batch_size=32,
# verbose=2,
# callbacks=[checkpoint])
# fcn32s.save('models/fcn_dibco.h5')
fcn32s = load_model('models/fcn_dibco.h5')
Predictions on test set¶
In [26]:
def crop(img, mult):
H, W, *c = img.shape
h = H - H%mult
w = W - W%mult
return img[:h,:w]
X_test = []
y_test = []
for line in test_imgs:
names = line.split()
test_img = m.imread(dir_dibco + names[0], mode='RGB')
test_seg = m.imread(dir_dibco + names[1], mode='L')
X_test.append(crop(test_img, 4))
y_ing = to_categorical(np.array([test_seg]))
y_test.append(crop(y_ing[0], 4))
In [27]:
y_predi = []
y_testi = []
for i, ing in enumerate(X_test):
y_pred = fcn32s.predict(np.asarray([ing]))
y_predi.append(np.argmax(y_pred, axis=3))
y_testi.append(np.argmax(y_test[i], axis=2))
In [28]:
for i, ing in enumerate(y_predi):
gtn = (y_testi[i] == 1).astype(int)
pn = (ing[0] == 1).astype(int)
gtp = (y_testi[i] == 0).astype(int)
pp = (ing[0] == 0).astype(int)
TP = (gtp*pp).sum()
TN = (gtn*pn).sum()
FP = (gtn*pp).sum()
FN = (gtp*pn).sum()
Precision = TP/(TP+FP)
Sensitivity = TP/(TP+FN)
print(Precision, Sensitivity)
In [29]:
# Calculate other metrics as described by DIBCO competition
In [30]:
for i in range(10):
img_is = X_test[i]
seg = y_predi[i][0]
segtest = y_testi[i]
fig = plt.figure(figsize=(20,40))
ax = fig.add_subplot(1,3,1)
ax.imshow(img_is, cmap='gray')
ax.set_title("original")
ax = fig.add_subplot(1,3,2)
ax.imshow(seg, cmap='gray')
ax.set_title("predicted class")
ax = fig.add_subplot(1,3,3)
ax.imshow(segtest, cmap='gray')
ax.set_title("true class")
plt.show()
m.imsave('images/dibco/'+str(i)+'_orig.png', img_is)
m.imsave('images/dibco/'+str(i)+'_pred_fcn.png', seg)
m.imsave('images/dibco/'+str(i)+'_segm.png', segtest)
print(img_is.shape, seg.shape, segtest.shape)
